{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "\n",
    "from helper import nn\n",
    "from helper import logistic_regression as lr\n",
    "\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# model\n",
    "<img style=\"float: left;\" src=\"../img/nn_model.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load weights and data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25, 401), (10, 26))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta1, theta2 = nn.load_weight('ex3weights.mat')\n",
    "\n",
    "theta1.shape, theta2.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">The original data is 90 degree off. So in data loading function, I use transpose to fix it.  \n",
    "However, the transposed data is not compatible to given parameters because those parameters are trained by original data. So for the sake of applying given parameters, I need to use original data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 401), (5000,))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = nn.load_data('ex3data1.mat',transpose=False)\n",
    "\n",
    "X = np.insert(X, 0, values=np.ones(X.shape[0]), axis=1)  # intercept\n",
    "\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# feed forward prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a1 = X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 25)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z2 = a1 @ theta1.T # (5000, 401) @ (25,401).T = (5000, 25)\n",
    "z2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "z2 = np.insert(z2, 0, values=np.ones(z2.shape[0]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 26)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a2 = lr.sigmoid(z2)\n",
    "a2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 10)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z3 = a2 @ theta2.T\n",
    "z3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  1.38245045e-04,   2.05540079e-03,   3.04012453e-03, ...,\n",
       "          4.91017499e-04,   7.74325818e-03,   9.96229459e-01],\n",
       "       [  5.87756717e-04,   2.85026516e-03,   4.14687943e-03, ...,\n",
       "          2.92311247e-03,   2.35616705e-03,   9.96196668e-01],\n",
       "       [  1.08683616e-04,   3.82659802e-03,   3.05855129e-02, ...,\n",
       "          7.51453949e-02,   6.57039547e-03,   9.35862781e-01],\n",
       "       ..., \n",
       "       [  6.27824726e-02,   4.50406476e-03,   3.54510925e-02, ...,\n",
       "          2.63669734e-03,   6.89448164e-01,   2.74369466e-05],\n",
       "       [  1.01908736e-03,   7.34360211e-04,   3.78558700e-04, ...,\n",
       "          1.45616578e-02,   9.75989758e-01,   2.33374461e-04],\n",
       "       [  5.90807037e-05,   5.41717668e-04,   2.58968308e-05, ...,\n",
       "          7.00508308e-03,   7.32814653e-01,   9.16696059e-02]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a3 = lr.sigmoid(z3)\n",
    "a3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = np.argmax(a3, axis=1) + 1  # numpy is 0 base index, +1 for matlab convention\n",
    "y_pred.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# accuracy\n",
    "so... accuracy on training data is not predicting the real world performance you know  \n",
    "All we can say is NN is very powerful model. Overfitting is easy here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       1.00      1.00      1.00       508\n",
      "          2       1.00      1.00      1.00       493\n",
      "          3       1.00      1.00      1.00       491\n",
      "          4       1.00      1.00      1.00       499\n",
      "          5       1.00      1.00      1.00       503\n",
      "          6       1.00      1.00      1.00       507\n",
      "          7       1.00      1.00      1.00       495\n",
      "          8       1.00      1.00      1.00       502\n",
      "          9       1.00      1.00      1.00       497\n",
      "         10       1.00      1.00      1.00       505\n",
      "\n",
      "avg / total       1.00      1.00      1.00      5000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_pred, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tf]",
   "language": "python",
   "name": "conda-env-tf-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
